import scrapy

from qiushi_Scrapy_58.items import QiushiScrapy58Item
class QiubaiSpider(scrapy.Spider):
    name = 'qiubai'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    # def parse(self, response):
    #     # 解析：作者的名称+段子内容
    #     div_list = response.xpath('//div[@class="col1 old-style-col1"]/div')
    #     # 在scrapy中不需要发送请求 直接使用xpath即可
    #     print(div_list)
    #     all_data = [] # 存储所有解析到的数据
    #     for div in div_list:
    #         # author = div.xpath('./div//h2/text()')[0].extract()
    #         author = div.xpath('./div//h2/text()').extract_first()
    #         content = div.xpath('./a/div/span//text()').extract()
    #         # 将列表转字符串
    #         content = ''.join(content)
    #         print(content)
    #
    #         div={
    #             'author': author,
    #             'content': content
    #         }
    #         # print(author,content)
    #         all_data.append(div)
    #
    #     return all_data

    # 基于管道的持久化存储


    def parse(self, response):
        # 解析：作者的名称+段子内容
        div_list = response.xpath('//div[@class="col1 old-style-col1"]/div')
        # 在scrapy中不需要发送请求 直接使用xpath即可
        print(div_list)
        all_data = [] # 存储所有解析到的数据
        for div in div_list:
            # author = div.xpath('./div//h2/text()')[0].extract()
            author = div.xpath('./div//h2/text()').extract_first()
            content = div.xpath('./a/div/span//text()').extract()
            # 将列表转字符串
            content = ''.join(content)
            print(content)
            item = QiushiScrapy58Item()
            item["author"] = author
            item["content"] = content

            # 将item提交给管道
            yield item